{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hill Climbing (with adaptive noise scaling)\n",
    "\n",
    "---\n",
    "\n",
    "   In this notebook, we train the Hill Climbing Agent with   \n",
    "**adaptive noise scaling** with OpenAI Gym's Cartpole environment."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Policy-Based Method\n",
    "\n",
    "In **policy-based methods**, instead of learning a **value function** that tells us what is     \n",
    "the expected sum of rewards given a state and an action, we learn directly the **policy function**    \n",
    "that maps state to action (select actions without using a value function)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Import the Necessary Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "import matplotlib.pyplot as plt\n",
    "#%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Define the Policy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id:  CartPole-v0\n",
      "self.env_specs[id]:  EnvSpec(CartPole-v0)\n",
      "my_env:  <CartPoleEnv<CartPole-v0>>\n",
      "observation space: Box(4,)\n",
      "action space: Discrete(2)\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "print('observation space:', env.observation_space)\n",
    "print('action space:', env.action_space)\n",
    "\n",
    "class Policy():\n",
    "    def __init__(self, s_size=4, a_size=2):\n",
    "        self.w = 1e-4*np.random.rand(s_size, a_size)  # weights for simple linear policy: state_space x action_space\n",
    "        \n",
    "    def forward(self, state):\n",
    "        x = np.dot(state, self.w)\n",
    "        return np.exp(x)/sum(np.exp(x))\n",
    "    \n",
    "    def act(self, state):\n",
    "        probs = self.forward(state)\n",
    "        #action = np.random.choice(2, p=probs) # option 1: stochastic policy\n",
    "        action = np.argmax(probs)              # option 2: deterministic policy\n",
    "        return action"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Train the Agent with Stochastic Policy Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id:  CartPole-v0\n",
      "self.env_specs[id]:  EnvSpec(CartPole-v0)\n",
      "my_env:  <CartPoleEnv<CartPole-v0>>\n",
      "Episode 100\tAverage Score: 175.24\n",
      "Environment solved in 113 episodes!\tAverage Score: 196.21\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('CartPole-v0')\n",
    "env.seed(0)\n",
    "np.random.seed(0)\n",
    "\n",
    "policy = Policy()\n",
    "\n",
    "def hill_climbing(n_episodes=1000, max_t=1000, gamma=1.0, print_every=100, noise_scale=1e-2):\n",
    "    \"\"\"Implementation of hill climbing with adaptive noise scaling.\n",
    "        \n",
    "    Params\n",
    "    ======\n",
    "        n_episodes (int): maximum number of training episodes\n",
    "        max_t (int): maximum number of timesteps per episode\n",
    "        gamma (float): discount rate\n",
    "        print_every (int): how often to print average score (over last 100 episodes)\n",
    "        noise_scale (float): standard deviation of additive noise\n",
    "    \"\"\"\n",
    "    scores_deque = deque(maxlen=100)\n",
    "    scores = []\n",
    "    best_R = -np.Inf\n",
    "    best_w = policy.w\n",
    "    for i_episode in range(1, n_episodes+1):\n",
    "        rewards = []\n",
    "        state = env.reset()\n",
    "        for t in range(max_t):\n",
    "            action = policy.act(state)\n",
    "            state, reward, done, _ = env.step(action)            \n",
    "            rewards.append(reward)\n",
    "            if done:\n",
    "                break \n",
    "        scores_deque.append(sum(rewards))\n",
    "        scores.append(sum(rewards))\n",
    "\n",
    "        discounts = [gamma**i for i in range(len(rewards)+1)]\n",
    "        R = sum([a*b for a,b in zip(discounts, rewards)])\n",
    "\n",
    "        if R >= best_R: # found better weights\n",
    "            best_R = R\n",
    "            best_w = policy.w\n",
    "            noise_scale = max(1e-3, noise_scale / 2)\n",
    "            policy.w += noise_scale * np.random.rand(*policy.w.shape) \n",
    "        else: # did not find better weights\n",
    "            noise_scale = min(2, noise_scale * 2)\n",
    "            policy.w = best_w + noise_scale * np.random.rand(*policy.w.shape)\n",
    "\n",
    "        if i_episode % print_every == 0:\n",
    "            print('Episode {}\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_deque)))\n",
    "        if np.mean(scores_deque)>=195.0:\n",
    "            print('Environment solved in {:d} episodes!\\tAverage Score: {:.2f}'.format(i_episode, np.mean(scores_deque)))\n",
    "            policy.w = best_w\n",
    "            break\n",
    "        \n",
    "    return scores\n",
    "            \n",
    "scores = hill_climbing()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Plot the Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEKCAYAAAAIO8L1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAIABJREFUeJzt3XuUnXV97/H3d1/mknsgkxBCYkABQWujnVJbLwfFWrEuqZ5apLZQ62m09XqOXS3ac6p2LdfqsV5aV3uosVDw1KZYo5XTcqwUPWIvUoPQiNwEBAkMySSBJDPJzOzL9/zxPM+eZ/Y8k5k94blMns9rrVmz59l7z/w2O/y++/v7/i7m7oiIiHSr5N0AEREpJgUIERFJpAAhIiKJFCBERCSRAoSIiCRSgBARkUQKECIikkgBQkREEilAiIhIolreDTgZ69at861bt+bdDBGRJeWOO+444O5D8z1uSQeIrVu3snv37rybISKypJjZowt5nIaYREQkkQKEiIgkUoAQEZFEChAiIpJIAUJERBKlFiDMbLOZfcPM7jWz75vZe8Prp5nZLWb2g/D72vC6mdmnzexBM9tjZi9Kq20iIjK/NDOIJvB+d78AeDHwTjO7ELgauNXdzwVuDX8GuBQ4N/zaDlyTYttERGQeqa2DcPcRYCS8fdTM7gU2AZcBF4cPuwH4f8Dvhtc/58EZqN82szVmtjH8Pae0f33wAGesHuCcoRULfs6xqSbX/+sjTEy1UmyZiBTVeWes5HUvODPVv5HJQjkz2wq8ELgd2BB1+u4+Ymbrw4dtAh6LPW1veG1GgDCz7QQZBlu2bEm13Vn5nV17+Jlnn87HfvHHF/yc2x8+xMe+ej8AZmm1TESK6nUvOHPpBwgzWwHsAt7n7kds7t4s6Q6fdcF9B7ADYHh4eNb9S9Fks83xRrvH5wSZw83veRkXnrkqjWaJSMmlOovJzOoEweHz7v6l8PI+M9sY3r8R2B9e3wtsjj39LOCJNNtXFK2202j2FiCmWkFsrFeVPohIOtKcxWTAtcC97v7J2F03AVeFt68CvhK7fmU4m+nFwOEy1B8AGq02U63eAkQzfHytqpnKIpKONIeYXgL8KvA9M7srvPZB4A+BL5jZ24AfAW8K77sZeC3wIHAMeGuKbSuUVttp9BwgggyiVlEGISLpSHMW0z+TXFcAuCTh8Q68M632FFmz7Uz2OMTUaAePryuDEJGUqHcpgJPKIFSDEJGUlDJAHJlocNsDoxwcm8y7Kbg7rbYz1WsGEQaUeqWUb6GIZKCUvcvDo+Nced2/8x97n867KTTbQSbQcwbRVgYhIukqZYCICruNVv7LKFqdANFbW6JZTKpBiEhaStm7RJ1qswABIsoceh9i0joIEUlXKQNENcwgmu3eOuU0RBlEr+sgGq021YpxgpXpIiInpZQBIvrUXYQMIqol9JpBNNuuNRAikqpSBoho9XGRMohei9SNVlv1BxFJVSl7mHqBitSLrUE0W64ZTCKSqlIGiE4G0eOn9jREGUSz7bTbCw9YzXabmtZAiEiKStnDRJ+8mz10yGmJt6HRw5BXo+WawSQiqSplgIhWHxdhiCleKO9lmKmpGoSIpKyUPUwngyjAEFO8UN5LwGq0VYMQkXSVM0BEReoCDDG12ovLIBrNtvZhEpFUlbKHMTNqFStIBhGrQfTQnqYyCBFJWSkDBASrqQtRpI4NK/VyJkSj1dZpciKSqjSPHL3OzPab2d2xazea2V3h1yPRSXNmttXMjsfu+/O02hWpVysFWUkdr0H0UqT2znoOEZE0pHnk6PXAnwKfiy64++XRbTP7BHA49viH3H1biu2ZoVa1Qq2khh5nMbXbGmISkVSllkG4+23AoaT7LNhh7peAnWn9/fnUKpViTHNdZA0iWAehISYRSU9ePczLgH3u/oPYtbPN7E4z+6aZvSztBtSrBSlSx9dB9FSk1joIEUlXmkNMJ3IFM7OHEWCLux80s58A/s7MnufuR7qfaGbbge0AW7ZsWXQDgiGm/DOIVmyYq7eFctrNVUTSlflHUDOrAW8Eboyuufukux8Mb98BPAScl/R8d9/h7sPuPjw0NLTodtQrlZ53UE3DzCGmHhbKaSW1iKQsjx7mVcB97r43umBmQ2ZWDW+fA5wLPJxmI2pVK8YspkVutdHQbq4ikrI0p7nuBP4NON/M9prZ28K73szs4vTLgT1m9h/AF4F3uHtigfuZUqtUCjGLadEL5VrazVVE0pVaDcLdr5jj+q8lXNsF7EqrLUnqVSvELKbF1iAabe3mKiLpKu1H0GAldbEyiJ5mMbW0DkJE0lXaAFErykrq1mKHmFxDTCKSqtL2MPWCTHNtLnY313ZbQ0wikqrSBohapVKIhXKtk9mLSdNcRSRFpe1hilKkbiximqu7h9t9l/btE5EMlLaHKco012izvmrFmFpgwIqCinZzFZE0lTdAFGWhXBggBuvVBWcQUWBTBiEiaSptD1OvVmgUIoNoU6sYfbWFb/3RySBUpBaRFJU2QARHjhYgg2g51YpRr9rCM4gwkGizPhFJU3kDRLU450H0mkFEw1IaYhKRNJW2h6kVZCV1qx1lEJUFr6SOAomGmEQkTeUNEFWjVYAMItq2u69a6WGIKapBlPbtE5EMlLaHKU6ROsggehti0iwmEUlfaXuYwhSpwxpEb0NMWgchIukrb4CoVmi2Hfd8g0QrXBHdV63QaCa3ZaLRYqLR6vwcZRrKIEQkTaXtYaJP33lv2NdoBesg6rUKk3NkEL+7aw/v3nln7DnRLCZlECKSntQODCq66NN3sOldfu3o1CCqFRpzFKkfPXhsxoyraB1EXdt9i0iK0jxy9Doz229md8eufdjMHjezu8Kv18bu+4CZPWhm95vZz6XVrkg0RTTvQnWzU6S2OYvU45NNJhvtGc8BZRAikq40P4JeD7wm4fqn3H1b+HUzgJldSHBW9fPC5/wvM0v1c320CjnvQnWr7dPTXE8UIJqztwXXOggRSVNqAcLdbwMOLfDhlwF/4+6T7v5D4EHgorTaBvEhpnwziEar3VkoN9cQ0/hUa8YaCa2DEJEs5NHDvMvM9oRDUGvDa5uAx2KP2RteS02UQTRyLlK3ommuteQMwt3DDGJ6FlNnHYRqECKSoqx7mGuAZwPbgBHgE+H1pLGSxJ7bzLab2W4z2z06OrrohkQZRN6rqYODf2zOldSTzTbNtncNMWk3VxFJX6YBwt33uXvL3dvAZ5keRtoLbI499CzgiTl+xw53H3b34aGhoUW3pTBF6labWqVC3xwZxPhkEyCxBqF1ECKSpkx7GDPbGPvxDUA0w+km4M1m1m9mZwPnAv+eZlui4ZkiFKk701wT2jI+2eo8LqqXRG3Wdt8ikqbU1kGY2U7gYmCdme0FPgRcbGbbCIaPHgHeDuDu3zezLwD3AE3gne7eSvq9z5RoiuhC9z9KS3yrjVbbOwEjMhZmEABTrXawTXk7msWkDEJE0pNagHD3KxIuX3uCx38U+Gha7ekWDTHlvZK6FdUgakFnH8xqmp7hOz41HSAmG22W9cUyCNUgRCRFpf0IOj3ElP8011ql0glY3XWIeAYR1SEaWkktIhkobQ8zPcSUfwYRbfcNzJrJNB4fYgrv00pqEclCaQNENH6f96ly8WmuMLsmMj4jgwjKMp29mFSDEJEUlbaHKcpWG83WdJEaZmcQY5PTtfrpISatgxCR9JU2QNTn+MSetWCzvsqMInVcYgbRDrbnMFOAEJH0lDZARFNJW7nPYmrPyCAmT1CDiGcQWgMhImkrbYCYXkldjK02+jsZxMz2zDWLSfUHEUlbaXuZokxz7a5BnHCIqTG9klozmEQkbeUNENViFKlbYQ2isw4ioUgd1SfiNQjt5CoiaSttL9P5xJ77NNegBtFZB5GQQZy+vC+4L1aD0AwmEUlbaQNEEaa5tttO24NsZq5pruNTTdYuCwJEVINoqgYhIhkobS9TK8A0186K6Eq8SD17q43TV8wMEI22ahAikr7SBogibNYXTbENahBzb7Vx2vIoQEyvpNY+TCKSttL2MkWYxdTsbNttJ1go1+oMMcVrEMogRCRtJQ4Q+W/WF9U/qvGtNmLtcXfGp5qsHKjRV63MWAeh0+REJG2l7WUqFaNi+a6kjtcgknZzPTbVwh2W99for1VmrIOoayW1iKSstAECmHE6Wx7iNYik3VyjRXLL+2v01ysz10FoiElEUpZagDCz68xsv5ndHbv2R2Z2n5ntMbMvm9ma8PpWMztuZneFX3+eVrvi6hXLdZprVIMIprnOXigXbbOxor/aNcTkmuYqIqlLs5e5HnhN17VbgOe7+wuAB4APxO57yN23hV/vSLFdHbVqJd8idWt6iKlWrVCx7gwiyBiW99Xor1djBwZpHYSIpC+1XsbdbwMOdV37mrtHmwt9Gzgrrb+/EPWq5bpZX7M9XaQO2lOZI4MIaxCdaa7azVVE0pfnx9BfB/5v7OezzexOM/ummb1srieZ2XYz221mu0dHR0+qAbVKvhlEVIOIsoG+WmXGVhszahC1inZzFZFM5dLLmNnvAU3g8+GlEWCLu78Q+G/AX5vZqqTnuvsOdx929+GhoaGTaketmm8NIhpOijKIvq4MYnwqHiCq07OYtJJaRDKQeYAws6uA1wFvcXcHcPdJdz8Y3r4DeAg4L+221KuVXIeYWrFprhBkEPEaRHyIKZ5dNJrazVVE0pdpL2NmrwF+F3i9ux+LXR8ys2p4+xzgXODhtNtTq1jOK6ln1yDiC/emh5iqM2oQjbZ2cxWR9NXS+sVmthO4GFhnZnuBDxHMWuoHbgnPU/52OGPp5cAfmFkTaAHvcPdDib/4GVTr6pCz1l2DqFetq0gdn8UUXyindRAikr7UAoS7X5Fw+do5HrsL2JVWW+ZSqxitHBfKNbtrELXqrCL1sr4qlYoFNYjm9EpqDTGJSNpK3cvUqpbrbq7N7hpEVwYxPtlkeX8tvG+6gN1otzXEJCKpK3WAqFcquZ4H0SlSx6a5dhepV4QBYsZWG1pJLSIZKHUvU5RprrU5FsoFGUQVoLMOwt3Daa6lfutEJAOl7mVqBZnmWp1jmuv4ZIvlfWEGEdYgomEp7eYqImlbcIAws5ea2VvD20NmdnZ6zcpGvSDTXGdkELGMZmwyOAsCguDRajsTjWCYSRmEiKRtQb2MmX2IYP1CtLleHfirtBqVlbyHmGbVIKoVpsI6AwQrqaMidXRmdbSBn4rUIpK2hX4MfQPwemAcwN2fAFam1ais5H0eRHcNIhhimrlQrjtARKurtVmfiKRtoQFiKtwWwwHMbHl6TcpO3udBdNcgZi+Ui89iCorVx8L9mTTEJCJpW2gv8wUz+wywxsx+A/gn4LPpNSsbuZ8H0Rliml2kbrbaTDTasSL1zAxCQ0wikrYFraR294+b2c8CR4Dzgd9391tSbVkGapV8F8pNb9YXbbUxPc11fCrcZiOc5to3qwahDEJE0jVvgAg30ftHd38VwYlwp4y8V1LP2u47tmPreGwnVwimucava4hJRNI2by/j7i3gmJmtzqA9maoVZSV1/DyIVrAYLn5YECQMMalILSIpW+hmfRPA98zsFsKZTADu/p5UWpWRes7TXGfVIKoV3IPAMTYrg4iGmJRBiEg2Fhog/iH8OqXUqhWaue7m2lWDCINAo+WdWkNns75ZAUIZhIika6FF6hvMrI/pU97ud/dGes3KRr1iNFqOuxOeT5GpaKvxaLQoKjxPNdudDGJ6L6bge3RGRF3bfYtIyhYUIMzsYuAG4BHAgM1mdpW735Ze09IXDdO0cjrjudl2ahXrBKcoS5hqtWcXqevKIEQkWwsdYvoE8Gp3vx/AzM4DdgI/kVbDshB1ssHuqNn//e7A1Bfenmq1GZ9KLlJH17UOQkTSttBxinoUHADc/QGC/ZhOyMyuM7P9ZnZ37NppZnaLmf0g/L42vG5m9mkze9DM9pjZi3p9Mb2KhmnymsnU6DoZLsogGs0239t7mJX9NVYP1mfc18kgNMQkIilbaC+z28yuNbOLw6/PAncs4HnXA6/punY1cKu7nwvcGv4McClwbvi1HbhmgW1btE4GkdNMpla73VkDAdM1iPGpJl+7Zx+vunBD59r0OggtlBORbCy0l/lN4PvAe4D3AvcA75jvSWGN4lDX5csI6hmE338hdv1zHvg2wbYeGxfYvkWJ1h/ktViu2fYZQ0V9Yad/2wMHOHy8waXPP6Nzn7baEJGsLbQGUQP+xN0/CZ3V1f2L/Jsb3H0EwN1HzGx9eH0T8FjscXvDayPxJ5vZdoIMgy1btiyyCYGoSJ3XVNdmy2dmEGEQ+Mpdj7O8r8rLzxvq3Nddg9A6CBFJ20J7mVuBwdjPgwQb9j2Tkj4Sz/po7+473H3Y3YeHhoYSnrJwnQwipyGmYBZTrAYRdvr3PXmUSy7YwEB9unJuZvRVK7EahDIIEUnXQgPEgLuPRT+Et5ct8m/ui4aOwu/7w+t7gc2xx50FPLHIv7Eg0Th+XkXq7hpEVIgGeO2PzR5d669VYkNMyiBEJF0L7WXG47OKzGwYOL7Iv3kTcFV4+yrgK7HrV4azmV4MHI6GotISn+aah2bXNNeo01/WV+Xi82dnR/31ChON8JAh1SBEJGULrUG8D/hbM3uCYNjnTODy+Z5kZjuBi4F1ZrYX+BDwhwTnS7wN+BHwpvDhNwOvBR4EjgFvXfjLWJxaztNcmy2fMVQUDTG98rnrZwwvRfpjizW0klpE0nbCAGFmPwk85u7fMbPnAm8H3gh8FfjhfL/c3a+Y465LEh7rwDvnbfEzqJ7zNNdm26nGOvr1q/oZqFd40/DmxMf3x4aglEGISNrm+xj6GWAqvP3TwAeBPwOeAnak2K5M5D2LqdVuz5iuum5FP3d/+Of4T+clF9/jNQrVIEQkbfMNMVXdPVrHcDmww913AbvM7K50m5a+6EyFRq4ZxMxM4ETTV/tnBAhlECKSrvk+hlbNLAoilwBfj9230PpFYXUyiLwCRFcNYj5RDaIa2+BPRCQt83XyO4FvmtkBgllL3wIws+cAh1NuW+qqnZXUeQ0xzc4gTiTa0VVrIEQkCycMEO7+UTO7FdgIfC0sJEOQebw77calLf8idZvl9YUnYtEsJ9UfRCQL8/ZO4b5I3dceSKc52Yqmuea21cZiMwjVH0QkA6X+KBplELkVqRdZg9BW3yKShVL3NPlPc/WeOvtoFpNmMIlIFsodIHKf5tqm2kNn31dTDUJEslPqnqae9zTXdq9DTKpBiEh2Sh0gpjfrK8Z5EPOJahDah0lEslDqnmb6TOq8jhz1njp7ZRAikqVSB4jpM6mDDOLqXXv4g/9zT2Z/f7E1CJ0mJyJZWPLbZZyMateZ1Lf/8BBDKxd7kmrvFluDqGsltYhkoNQfRbuL1KNHJ5lotDL7+61eaxDhGREaYhKRLJQ6QASb3gVDPcemmoxNNjMNEM229zRltV/TXEUkQ5kPMZnZ+cCNsUvnAL8PrAF+AxgNr3/Q3W9Ouz31SoVGyzlwNDj24nimAaK9qFlM2qxPRLKQeYBw9/uBbQBmVgUeB75McMTop9z941m2p1Y1mq02o2MTAJ0zn7PQaw1CC+VEJEt59zSXAA+5+6N5NaBWMZptZ/ToJAATU9lkEO22497bvkoaYhKRLOXd07yZ4MyJyLvMbI+ZXWdma7NoQL1aodFqTweIZjYBohEuzuul4Kx1ECKSpdwChJn1Aa8H/ja8dA3wbILhpxHgE3M8b7uZ7Taz3aOjo0kP6UkwxOTsDwNEo+WddRFpaoVTaxc1i0krqUUkA3n2NJcC33X3fQDuvs/dW+7eBj4LXJT0JHff4e7D7j48NDR00o2oVSo02tMZBMBEM/0AEa296KkG0TkwSBmEiKQvzwBxBbHhJTPbGLvvDcDdWTSiHmYQ8QBxPIM6RKvVe4DQgUEikqVcVlKb2TLgZ4G3xy5/zMy2AQ480nVfaqoVo9V2RsdiGUQGU12jGkR1EesgNMQkIlnIJUC4+zHg9K5rv5pHW+JF6sF6leONViYBorWIIabObq7KIEQkA6X/KFqrGo1WmwNjk2w+bRDIZrFccxFDTFoHISJZKn1PU6tUODA2RaPlbDltGZDNYrlOBrGoaa6lf9tEJAOl72nqVWPk8HEANocBIpMMIqpB9LhQ7tUXbmD4WZksERGRkiv1dt8wnUEAbF4bZRBZBIjeh5jMjB1XDqfVJBGRGUqfQcSHeKaHmIpZgxARyVLpA0S84Ls5ywCxiBqEiEiWSh8gok/wA/UK61b0ARktlFtEDUJEJEul752iDGJoZT/L+oKSTCZbbYRDTDo+VESKqvQBItosb2hFf2caaTYZRO+b9YmIZKn0ASKqAaxfOUClYvTXKpls+d1QDUJECq70AaJemR5iAhioVzM5NEg1CBEputL3TtEn+ChADNarmayk1jRXESm60geIeJEagtlM2ayk1hCTiBRb6QNELVakhnCIqaArqUVEsqQAMSuDqGaSQUQ1CJ3tICJFVfreqZ5Qg5jMsAahaa4iUlSlDxCrB+sM1qusW6EahIhIXG67uZrZI8BRoAU03X3YzE4DbgS2Ehw7+kvu/lSa7XjLTz2LVz53fecwnsG+bGsQyiBEpKjyziBe4e7b3D3aw/pq4FZ3Pxe4Nfw5VYN9Vc4ZWtH5eaCWUQ2iFQxj1VWDEJGCKlrvdBlwQ3j7BuAXsm7AQF9G6yCiDEJDTCJSUHkGCAe+ZmZ3mNn28NoGdx8BCL+vz7pRAzVNcxURgXxPlHuJuz9hZuuBW8zsvoU8KQwm2wG2bNnyjDdqsK+SSYDQZn0iUnS5ZRDu/kT4fT/wZeAiYJ+ZbQQIv+9PeN4Odx929+GhoaFnvF0DtSrNttNopTvMNL3dd9FG+UREArn0Tma23MxWRreBVwN3AzcBV4UPuwr4StZtG+yrAumfKtdstzGDijIIESmovIaYNgBfNrOoDX/t7l81s+8AXzCztwE/At6UdcP660GAON5osXKgntrfabZd9QcRKbRcAoS7Pwz8eML1g8Al2bdo2mAYINJeTd1qu7bZEJFCUw/VZaAeniqX9hBTSxmEiBSbAkSXKIPIogahNRAiUmQKEF0GohpEyqfKqQYhIkWnANElChATzXRrEEcnmqzoz3MZiojIiSlAdOnUIFLOIA6OTXJ6uIOsiEgRKUB06cxiaqYdIKY4fXlfqn9DRORkKEB0yaoGcXBcGYSIFJsCRJfB2EK5tLTbzqHxKdatUAYhIsWlANGlU6ROcaHc08cbtB1O0xCTiBSYAkSX/lr6C+UOjk0CaIhJRApNAaJLpWL01ypMnmSAuO2BUW57YDTxvgNjUwCsUwYhIgWmifgJBvtO/tjRP/rH+6kYvPy82VuSHxoPAoQyCBEpMmUQCQbryafK3f/kUb5+374F/Y6RwxPsOzKZeN/B8WiISRmEiBSXAkSCgXqV4wlF6j/9xoP89t/umff5U802B8YmGR2b7JwcF3dgbAozWLtMAUJEiksBIsHAHBnEyNPHOTQ+xbGp5gmfv+/IBBBs6R1lC3EHxyZZu6xPx42KSKEpQCQYqCefSz1yOOj4n3j6+Amf/2QYIAD2JwwzHRrXKmoRKT4FiARJNYhW2zuZwd6nThwgokAC09lE3MGxKdUfRKTwMg8QZrbZzL5hZvea2ffN7L3h9Q+b2eNmdlf49dqs2xYJahAzA8TBsUmaYT3h8fkyiMPT9ycVqg+MT3L6cs1gEpFiy2OaaxN4v7t/18xWAneY2S3hfZ9y94/n0KYZggxiZpE6nhXMN8T0xNMTwe9otpRBiMiSlXmAcPcRYCS8fdTM7gU2Zd2OE+mvV2Zt1hcPEI/PM8T05OEJNq0d5OljDfYfnRkgGq02h483lEGISOHlWoMws63AC4Hbw0vvMrM9Znadma2d4znbzWy3me0eHU1eqXyyBuvVWdt9R8NGz1m/Yt4hppEjE2xcPcCGVf2zhpie6iySUwYhIsWWW4AwsxXALuB97n4EuAZ4NrCNIMP4RNLz3H2Huw+7+/DQ0OxVys+EgXp1dgZxZIK+aoUf27SaJ56ePWwU9+Th45yxaoANqwZmDTF1ttlQgBCRgsslQJhZnSA4fN7dvwTg7vvcveXubeCzwEV5tA3CGkSzjfv0IrcnD09wxuoBzlo7yJNHJmi2knd7bbTa7D86OWcGEa2LOE1DTCJScHnMYjLgWuBed/9k7PrG2MPeANydddsiA/UKrbbTaE0HiJGngwCxac0grbbPWOsQN3p0EnfYuGaQ9SsHODg+SSMWTA6OaYhJRJaGPGYxvQT4VeB7ZnZXeO2DwBVmtg1w4BHg7Tm0DYidCdFs0Rdu/z1y5Dgv2rKWM9cMAkGh+qy1y2Y9Nypmn7F6AHdwhwNjk2xcHTzv4Hi0k6syCBEptjxmMf0zkLTHxM1Zt2UunQAx1WLVQJ1229l3OOjkN60NOvonDicXqp8MA8TG1QO0w3UT+47EAsTYJLWKsWpQG+mKSLGpl0ow2HWq3KFjU0y12mwMh5hg7qmuI2Hg2LhqkGYrChDTw1HRGohgpE1EpLgUIBIMdJ1L/WRs2GigXmXdir45p7qOHA4Wya0arLG+FQwj7Y8HiPFJFahFZEnQXkwJBvuC/yzRfkwjsWEjgDPXDPL4HFNdnzw8wcY1A5gZpy/vp1qxGTOZDoxNaYqriCwJChAJBmozM4ho2OiMMEBsWjPI408dS3zuyOHjnUBSrRhDK/pnDDFpJ1cRWSoUIBIM9EU1iOkMolaxzsyjTWsGefzp4zPWSUSePDzBGasGOz9vWNXPvqPTGcTBsUkdNSoiS4ICRIIog5iI1SA2rBqgEh7ws2ntIBONduds6Uir7ewLF8lF1q8a6NQgjk+1GJ9qaQ2EiCwJChAJBvtmzmIaOXycM9dMd/rRWojuLTcOhEeMnhELEMFq6uBxnbOoNcQkIkuAAkSCgXrwnyU+i+mM1dPDRp2prk/PrENExex4MNmwcoCnjjWYbLY6GYd2chWRpUABIsGyejD790eHjuHujByemDFsdFa4WK77ZLlox9eZNYjgefuPTHLfyFFA22yIyNKgdRAJVi8au0UWAAAIRklEQVSr86oLNvCZbz7EOeuWM9lsc8aq6QCxerDO8r4qX/ru42xaM8grnrue7zxyiBv+9VGArhpEkC380737+NhX7+f5m1bxvDNXZ/uCREQWQQFiDp++Yhtv3vFtfmfXHmBmp29mvP/V5/OZ2x7iNz//XaoVo9V2Vg/W+c2Ln83aWI0hyiD+4O/vYdOaQa77tZ/s7O8kIlJkChBzWNZX49qrfpI3XvMvPHbo+IzCM8Cvv/RsrvzpZ/GtBw/wzftHGd66llddsKGzCjsSBYhVA3Wuf+tFrF858/eIiBSVAsQJDK3s53O//lP87397lAvPXDXr/lq1wivOX88rzl8/5+9Yu6zOu1/5HC65YAPPWb8izeaKiDyjLGmx11IxPDzsu3fvzrsZIiJLipnd4e7D8z1Og+EiIpJIAUJERBIVLkCY2WvM7H4ze9DMrs67PSIiZVWoAGFmVeDPgEuBCwmOIb0w31aJiJRToQIEcBHwoLs/7O5TwN8Al+XcJhGRUipagNgEPBb7eW94TUREMla0AJF0UPOMebhmtt3MdpvZ7tHR0YyaJSJSPkULEHuBzbGfzwKeiD/A3Xe4+7C7Dw8NDWXaOBGRMinUQjkzqwEPAJcAjwPfAX7Z3b8/x+NHgUd7/DPrgAMn086COlVfF5y6r02va+k5VV7bs9x93k/Yhdpqw92bZvYu4B+BKnDdXMEhfHzPKYSZ7V7ICsKl5lR9XXDqvja9rqXnVH5tSQoVIADc/Wbg5rzbISJSdkWrQYiISEGUMUDsyLsBKTlVXxecuq9Nr2vpOZVf2yyFKlKLiEhxlDGDEBGRBShNgDiVNgE0s81m9g0zu9fMvm9m7w2vn2Zmt5jZD8Lva/Nu62KYWdXM7jSzvw9/PtvMbg9f141m1jff7ygaM1tjZl80s/vC9+2nT6H367+G/w7vNrOdZjawFN8zM7vOzPab2d2xa4nvkQU+HfYne8zsRfm1PD2lCBCn4CaATeD97n4B8GLgneHruRq41d3PBW4Nf16K3gvcG/v5fwKfCl/XU8DbcmnVyfkT4Kvu/lzgxwle35J/v8xsE/AeYNjdn08wPf3NLM337HrgNV3X5nqPLgXODb+2A9dk1MZMlSJAcIptAujuI+7+3fD2UYLOZhPBa7ohfNgNwC/k08LFM7OzgJ8H/iL82YBXAl8MH7LkXpeZrQJeDlwL4O5T7v40p8D7FaoBg+FC12XACEvwPXP324BDXZfneo8uAz7ngW8Da8xsYzYtzU5ZAsQpuwmgmW0FXgjcDmxw9xEIgggw92HZxfXHwO8A7fDn04Gn3b0Z/rwU37tzgFHgL8Ohs78ws+WcAu+Xuz8OfBz4EUFgOAzcwdJ/zyJzvUenbJ8SV5YAMe8mgEuRma0AdgHvc/cjebfnZJnZ64D97n5H/HLCQ5fae1cDXgRc4+4vBMZZgsNJScIx+cuAs4EzgeUEwy/dltp7Np9T4d/lvMoSIObdBHCpMbM6QXD4vLt/Kby8L0pzw+/782rfIr0EeL2ZPUIwDPhKgoxiTTh8AUvzvdsL7HX328Ofv0gQMJb6+wXwKuCH7j7q7g3gS8DPsPTfs8hc79Ep16ckKUuA+A5wbjizoo+giHZTzm1atHBc/lrgXnf/ZOyum4CrwttXAV/Jum0nw90/4O5nuftWgvfo6+7+FuAbwC+GD1uKr+tJ4DEzOz+8dAlwD0v8/Qr9CHixmS0L/11Gr21Jv2cxc71HNwFXhrOZXgwcjoaiTiWlWShnZq8l+DQabQL40ZybtGhm9lLgW8D3mB6r/yBBHeILwBaC/3Hf5O7dRbclwcwuBn7b3V9nZucQZBSnAXcCv+Luk3m2r1dmto2g8N4HPAy8leAD2pJ/v8zsI8DlBLPr7gT+C8F4/JJ6z8xsJ3AxwY6t+4APAX9HwnsUBsM/JZj1dAx4q7vvzqPdaSpNgBARkd6UZYhJRER6pAAhIiKJFCBERCSRAoSIiCRSgBARkUQKEFJKZtYys7tiXydc2Wxm7zCzK5+Bv/uIma1bxPN+zsw+bGZrzUxH8komCncmtUhGjrv7toU+2N3/PM3GLMDLCBafvRz4l5zbIiWhACESE27zcSPwivDSL7v7g2b2YWDM3T9uZu8B3kGwMOwed3+zmZ0GXEewMd8xYLu77zGz04GdwBDw78T28DGzXyHYKruPYJHjb7l7q6s9lwMfCH/vZcAG4IiZ/ZS7vz6N/wYiEQ0xSVkNdg0xXR6774i7X0SwUvaPE557NfBCd38BQaAA+AhwZ3jtg8DnwusfAv453KTvJoIVuZjZBQSrj18SZjIt4C3df8jdbyTYt+lud/8x4O7wbys4SOqUQUhZnWiIaWfs+6cS7t8DfN7M/o5gKwaAlwL/GcDdv25mp5vZaoIhoTeG1//BzJ4KH38J8BPAd4JdGxhk7s36zgUeCm8vC88AEUmdAoTIbD7H7cjPE3T8rwf+h5k9jxNv/5z0Owy4wd0/cKKGmNlugr2BamZ2D7DRzO4C3u3u3zrxyxA5ORpiEpnt8tj3f4vfYWYVYLO7f4PgYKM1wArgNsIhonCjwQPhGR3x65cC0bnTtwK/aGbrw/tOM7NndTfE3YeBfyCoP3wM+D1336bgIFlQBiFlNRh+Eo981d2jqa79ZnY7wQeoK7qeVwX+Khw+MoJzl58Oi9h/aWZ7CIrU0RbRHwF2mtl3gW8S7AiKu99jZv8d+FoYdBrAO4FHE9r6IoJi9m8Bn0y4XyQV2s1VJCacxTTs7gfybotI3jTEJCIiiZRBiIhIImUQIiKSSAFCREQSKUCIiEgiBQgREUmkACEiIokUIEREJNH/B/UerFph3iEUAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xe1ce2e8>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "plt.plot(np.arange(1, len(scores)+1), scores)\n",
    "plt.ylabel('Score')\n",
    "plt.xlabel('Episode #')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Watch a Smart Agent!\n",
    "\n",
    "Each step of the Smart Agent animation includes the cl of the following _action_, _state_ and _reward_.     \n",
    "This also includes a slight _delay_ (0.1 sec) for better observation of the animation, and  further rendering."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "id:  CartPole-v0\n",
      "self.env_specs[id]:  EnvSpec(CartPole-v0)\n",
      "my_env:  <CartPoleEnv<CartPole-v0>>\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAD8CAYAAAB9y7/cAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEpRJREFUeJzt3X+s3fV93/Hnq5hAlmQ1hAtybTOTxltDp8XQO+KIaaKQtsB+mErNBK0aFCFdJhEpUaOt0ElrIg2pldawRetQ3ELjTFkII8lwEW3KHKIqfwRiJ45j41BuEie+tYfNAiRZNDaT9/64nxtOzfG9x/fc6+v7yfMhHZ3v93M+3+95f+Dwut/7ud8PJ1WFJKk/P7XSBUiSlocBL0mdMuAlqVMGvCR1yoCXpE4Z8JLUqWUL+CTXJ3k6yXSSO5frfSRJw2U57oNPcg7wV8AvATPAl4BbquqpJX8zSdJQy3UFfxUwXVXfrKr/CzwAbFum95IkDbFmmc67Hjg8sD8DvO1UnS+66KLatGnTMpUiSavPoUOHeO655zLOOZYr4IcV9TfmgpJMAVMAl156Kbt3716mUiRp9ZmcnBz7HMs1RTMDbBzY3wAcGexQVdurarKqJicmJpapDEn6ybVcAf8lYHOSy5K8BrgZ2LlM7yVJGmJZpmiq6kSS9wCfBc4B7q+qA8vxXpKk4ZZrDp6qehR4dLnOL0manytZJalTBrwkdcqAl6ROGfCS1CkDXpI6ZcBLUqcMeEnqlAEvSZ0y4CWpUwa8JHXKgJekThnwktQpA16SOmXAS1KnDHhJ6pQBL0mdMuAlqVMGvCR1aqyv7EtyCPg+8DJwoqomk1wIfBLYBBwC/kVVPT9emZKk07UUV/C/WFVbqmqy7d8J7KqqzcCuti9JOsOWY4pmG7Cjbe8AblqG95AkLWDcgC/gL5LsSTLV2i6pqqMA7fniMd9DkrQIY83BA1dX1ZEkFwOPJfn6qAe2HwhTAJdeeumYZUiSTjbWFXxVHWnPx4DPAFcBzyZZB9Cej53i2O1VNVlVkxMTE+OUIUkaYtEBn+R1Sd4wtw38MrAf2Anc2rrdCjw8bpGSpNM3zhTNJcBnksyd579W1Z8n+RLwYJLbgO8A7xy/TEnS6Vp0wFfVN4G3Dmn/X8B14xQlSRqfK1klqVMGvCR1yoCXpE4Z8JLUKQNekjplwEtSpwx4SeqUAS9JnTLgJalTBrwkdcqAl6ROGfCS1CkDXpI6ZcBLUqcMeEnqlAEvSZ0y4CWpUwa8JHXKgJekTi0Y8EnuT3Isyf6BtguTPJbkmfZ8QWtPkg8nmU6yL8mVy1m8JOnURrmC/yhw/UltdwK7qmozsKvtA9wAbG6PKeDepSlTknS6Fgz4qvpL4LsnNW8DdrTtHcBNA+0fq1lfBNYmWbdUxUqSRrfYOfhLquooQHu+uLWvBw4P9Jtpba+SZCrJ7iS7jx8/vsgyJEmnstR/ZM2QthrWsaq2V9VkVU1OTEwscRmSpMUG/LNzUy/t+VhrnwE2DvTbABxZfHmSpMVabMDvBG5t27cCDw+0v6vdTbMVeHFuKkeSdGatWahDkk8A1wAXJZkBfhf4PeDBJLcB3wHe2bo/CtwITAM/BN69DDVLkkawYMBX1S2neOm6IX0LuGPcoiRJ43MlqyR1yoCXpE4Z8JLUKQNekjplwEtSpwx4SeqUAS9JnTLgJalTBrwkdcqAl6ROGfCS1CkDXpI6ZcBLUqcMeEnqlAEvSZ0y4CWpUwa8JHXKgJekTi0Y8EnuT3Isyf6Btg8k+eske9vjxoHX7koyneTpJL+yXIVLkuY3yhX8R4Hrh7TfU1Vb2uNRgCSXAzcDP9+O+c9JzlmqYiVJo1sw4KvqL4Hvjni+bcADVfVSVX0LmAauGqM+SdIijTMH/54k+9oUzgWtbT1weKDPTGt7lSRTSXYn2X38+PExypAkDbPYgL8X+FlgC3AU+IPWniF9a9gJqmp7VU1W1eTExMQiy5AkncqiAr6qnq2ql6vqR8Af8co0zAywcaDrBuDIeCVKkhZjUQGfZN3A7q8Cc3fY7ARuTnJeksuAzcCT45UoSVqMNQt1SPIJ4BrgoiQzwO8C1yTZwuz0yyHgdoCqOpDkQeAp4ARwR1W9vDylS5Lms2DAV9UtQ5rvm6f/3cDd4xQlSRqfK1klqVMGvCR1yoCXpE4Z8JLUKQNekjplwEtSpxa8TVL6SbFn++1D239h6iNnuBJpaXgFL0mdMuAlqVMGvCR1yoCX5uH8u1YzA16SOmXAS1KnDHhJ6pQBL0mdMuAlqVMGvCR1yoCXpE4tGPBJNiZ5PMnBJAeSvLe1X5jksSTPtOcLWnuSfDjJdJJ9Sa5c7kFIkl5tlCv4E8D7q+otwFbgjiSXA3cCu6pqM7Cr7QPcAGxujyng3iWvWpK0oAUDvqqOVtWX2/b3gYPAemAbsKN12wHc1La3AR+rWV8E1iZZt+SVS5LmdVpz8Ek2AVcATwCXVNVRmP0hAFzcuq0HDg8cNtPaTj7XVJLdSXYfP3789CuXJM1r5IBP8nrgU8D7qup783Ud0lavaqjaXlWTVTU5MTExahmSpBGNFPBJzmU23D9eVZ9uzc/OTb2052OtfQbYOHD4BuDI0pQrSRrVKHfRBLgPOFhVHxp4aSdwa9u+FXh4oP1d7W6arcCLc1M5kqQzZ5Sv7Lsa+E3ga0n2trbfAX4PeDDJbcB3gHe21x4FbgSmgR8C717SiiVJI1kw4KvqCwyfVwe4bkj/Au4Ysy5J0phcySpJnTLgJalTBrwkdcqAl6ROGfCS1CkDXpI6ZcBLUqcMeAnYs/32lS5BWnIGvCR1yoCXpE4Z8JLUKQNekjplwEtSpwx4SeqUAS9JnTLgJalTBrwkdcqAl6ROjfKl2xuTPJ7kYJIDSd7b2j+Q5K+T7G2PGweOuSvJdJKnk/zKcg5AkjTcKF+6fQJ4f1V9OckbgD1JHmuv3VNV/36wc5LLgZuBnwd+BvgfSf5uVb28lIVLkua34BV8VR2tqi+37e8DB4H18xyyDXigql6qqm8B08BVS1GsJGl0pzUHn2QTcAXwRGt6T5J9Se5PckFrWw8cHjhshvl/IEiSlsHIAZ/k9cCngPdV1feAe4GfBbYAR4E/mOs65PAacr6pJLuT7D5+/PhpFy5Jmt9IAZ/kXGbD/eNV9WmAqnq2ql6uqh8Bf8Qr0zAzwMaBwzcAR04+Z1Vtr6rJqpqcmJgYZwySpCFGuYsmwH3Awar60ED7uoFuvwrsb9s7gZuTnJfkMmAz8OTSlSxJGsUod9FcDfwm8LUke1vb7wC3JNnC7PTLIeB2gKo6kORB4Clm78C5wztoJOnMWzDgq+oLDJ9Xf3SeY+4G7h6jLknSmFzJKkmdMuAlqVMGvCR1yoCXTuEXpj6y0iVIYzHgJalTBrwkdcqAl6ROGfCS1CkDXpI6ZcBLUqcMeEnqlAEvSZ0y4CWpUwa8upVk5MdyHC+tNANekjo1yhd+SD8R/vTI1I+3/9nPbF/BSqSl4RW8xN8M92H70mpkwEtSp0b50u3zkzyZ5KtJDiT5YGu/LMkTSZ5J8skkr2nt57X96fb6puUdgiRpmFGu4F8Crq2qtwJbgOuTbAV+H7inqjYDzwO3tf63Ac9X1ZuBe1o/6ax28py7c/DqwShful3AD9ruue1RwLXAr7f2HcAHgHuBbW0b4CHgPyVJO490Vpq8fTvwSqh/cOVKkZbMSHfRJDkH2AO8GfhD4BvAC1V1onWZAda37fXAYYCqOpHkReCNwHOnOv+ePXu8l1irmp9fnY1GCviqehnYkmQt8BngLcO6tedhn/RXXb0nmQKmAC699FK+/e1vj1SwNKozGbr+gqqlNjk5OfY5Tusumqp6Afg8sBVYm2TuB8QG4EjbngE2ArTXfxr47pBzba+qyaqanJiYWFz1kqRTGuUumol25U6S1wLvAA4CjwO/1rrdCjzctne2fdrrn3P+XZLOvFGmaNYBO9o8/E8BD1bVI0meAh5I8u+ArwD3tf73Af8lyTSzV+43L0PdkqQFjHIXzT7giiHt3wSuGtL+f4B3Lkl1kqRFcyWrJHXKgJekThnwktQp/3fB6pY3b+knnVfwktQpA16SOmXAS1KnDHhJ6pQBL0mdMuAlqVMGvCR1yoCXpE4Z8JLUKQNekjplwEtSpwx4SeqUAS9JnTLgJalTo3zp9vlJnkzy1SQHknywtX80ybeS7G2PLa09ST6cZDrJviRXLvcgJEmvNsr/D/4l4Nqq+kGSc4EvJPmz9tq/qqqHTup/A7C5Pd4G3NueJUln0IJX8DXrB2333PaY75sUtgEfa8d9EVibZN34pUqSTsdIc/BJzkmyFzgGPFZVT7SX7m7TMPckOa+1rQcODxw+09okSWfQSAFfVS9X1RZgA3BVkr8P3AX8HPAPgQuB327dM+wUJzckmUqyO8nu48ePL6p4SdKpndZdNFX1AvB54PqqOtqmYV4C/gS4qnWbATYOHLYBODLkXNurarKqJicmJhZVvCTp1Ea5i2Yiydq2/VrgHcDX5+bVkwS4CdjfDtkJvKvdTbMVeLGqji5L9ZKkUxrlLpp1wI4k5zD7A+HBqnokyeeSTDA7JbMX+Jet/6PAjcA08EPg3UtftiRpIQsGfFXtA64Y0n7tKfoXcMf4pUmSxuFKVknqlAEvSZ0y4CWpUwa8JHXKgJekThnwktQpA16SOmXAS1KnDHhJ6pQBL0mdMuAlqVMGvCR1yoCXpE4Z8JLUKQNekjplwEtSpwx4SeqUAS9JnTLgJalTIwd8knOSfCXJI23/siRPJHkmySeTvKa1n9f2p9vrm5andEnSfE7nCv69wMGB/d8H7qmqzcDzwG2t/Tbg+ap6M3BP6ydJOsNGCvgkG4B/Avxx2w9wLfBQ67IDuKltb2v7tNeva/0lSWfQmhH7/QfgXwNvaPtvBF6oqhNtfwZY37bXA4cBqupEkhdb/+cGT5hkCphquy8l2b+oEZz9LuKksXei13FBv2NzXKvL30kyVVXbF3uCBQM+yT8FjlXVniTXzDUP6VojvPZKw2zR29t77K6qyZEqXmV6HVuv44J+x+a4Vp8ku2k5uRijXMFfDfzzJDcC5wN/m9kr+rVJ1rSr+A3AkdZ/BtgIzCRZA/w08N3FFihJWpwF5+Cr6q6q2lBVm4Cbgc9V1W8AjwO/1rrdCjzctne2fdrrn6uqV13BS5KW1zj3wf828FtJppmdY7+vtd8HvLG1/xZw5wjnWvSvIKtAr2PrdVzQ79gc1+oz1tjixbUk9cmVrJLUqRUP+CTXJ3m6rXwdZTrnrJLk/iTHBm/zTHJhksfaKt/HklzQ2pPkw22s+5JcuXKVzy/JxiSPJzmY5ECS97b2VT22JOcneTLJV9u4Ptjau1iZ3euK8ySHknwtyd52Z8mq/ywCJFmb5KEkX2//rb19Kce1ogGf5BzgD4EbgMuBW5JcvpI1LcJHgetParsT2NVW+e7ilb9D3ABsbo8p4N4zVONinADeX1VvAbYCd7R/N6t9bC8B11bVW4EtwPVJttLPyuyeV5z/YlVtGbglcrV/FgH+I/DnVfVzwFuZ/Xe3dOOqqhV7AG8HPjuwfxdw10rWtMhxbAL2D+w/Daxr2+uAp9v2R4BbhvU72x/M3iX1Sz2NDfhbwJeBtzG7UGZNa//x5xL4LPD2tr2m9ctK136K8WxogXAt8Aiza1JW/bhajYeAi05qW9WfRWZvOf/Wyf/cl3JcKz1F8+NVr83gitjV7JKqOgrQni9u7atyvO3X9yuAJ+hgbG0aYy9wDHgM+AYjrswG5lZmn43mVpz/qO2PvOKcs3tcMLtY8i+S7Gmr4GH1fxbfBBwH/qRNq/1xktexhONa6YAfadVrR1bdeJO8HvgU8L6q+t58XYe0nZVjq6qXq2oLs1e8VwFvGdatPa+KcWVgxflg85Cuq2pcA66uqiuZnaa4I8k/nqfvahnbGuBK4N6qugL438x/W/lpj2ulA35u1eucwRWxq9mzSdYBtOdjrX1VjTfJucyG+8er6tOtuYuxAVTVC8Dnmf0bw9q28hqGr8zmLF+ZPbfi/BDwALPTND9ecd76rMZxAVBVR9rzMeAzzP5gXu2fxRlgpqqeaPsPMRv4SzaulQ74LwGb21/6X8PsStmdK1zTUhhczXvyKt93tb+GbwVenPtV7GyTJMwuWjtYVR8aeGlVjy3JRJK1bfu1wDuY/cPWql6ZXR2vOE/yuiRvmNsGfhnYzyr/LFbV/wQOJ/l7rek64CmWclxnwR8abgT+itl50H+z0vUsov5PAEeB/8fsT9jbmJ3L3AU8054vbH3D7F1D3wC+BkyudP3zjOsfMfvr3z5gb3vcuNrHBvwD4CttXPuBf9va3wQ8CUwD/w04r7Wf3/an2+tvWukxjDDGa4BHehlXG8NX2+PAXE6s9s9iq3ULsLt9Hv87cMFSjsuVrJLUqZWeopEkLRMDXpI6ZcBLUqcMeEnqlAEvSZ0y4CWpUwa8JHXKgJekTv1/446C/8sYTu8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x139cb6d8>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "env = gym.make('CartPole-v0')\n",
    "state = env.reset()\n",
    "img = plt.imshow(env.render(mode='rgb_array'))\n",
    "#for t in range(200):\n",
    "for t in range(2000):\n",
    "    action = policy.act(state)\n",
    "    env.render()\n",
    "    time.sleep(0.1)\n",
    "    state, reward, done, _ = env.step(action)\n",
    "    if done:\n",
    "        break \n",
    "\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
